import scrapy
from tutorial.items import DoubanItem


class DoubanMovieSpider(scrapy.Spider):
    name ="douban"
    #allowed_domains = ["movie.douban.com"] #为什么不要http
    start_urls = ["https://movie.douban.com/top250"]

    def parse(self, response):


        for sel in response.xpath('//div[@class="item"]'):
            item = DoubanItem()
            item['rank'] = sel.xpath('div/em/text()').extract()[0]
            titles = sel.xpath('div/div[@class="hd"]/a/span/text()').extract()
            name = ''
            for title in titles:
                name += title.strip()

            item['name'] = name
            item['rating'] = sel.xpath('div/div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]/text()').extract()[0]
            yield item

        next_page = response.xpath('//span[@class="next"]/a/@href').extract()[0]
        url = 'https://movie.douban.com/top250' + next_page
        print(url)
        if next_page:
            yield scrapy.Request(url,callback=self.parse)
